{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean: 180151.233516,  std: 76670.249891\n",
      "(1456, 80) (1459, 80)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd \n",
    "pd.set_option('display.max_columns', 100)\n",
    "import numpy as np \n",
    "import seaborn as sns \n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.stats import norm, skew\n",
    "from scipy import stats\n",
    "\n",
    "train_set = pd.read_csv('train.csv')\n",
    "test_set = pd.read_csv('test.csv')\n",
    "\n",
    "train_id = train_set['Id']\n",
    "test_id = test_set['Id']\n",
    "train_set.head()\n",
    "\n",
    "# 移除离群点\n",
    "train_set.drop(train_set[train_set['GrLivArea'] > 4000].index, axis=0, inplace=True)\n",
    "\n",
    "(mean, std) = norm.fit(train_set['SalePrice'])\n",
    "print('mean: %f,  std: %f' % (mean, std))\n",
    "\n",
    "# apply log(1+x) to column SalePrice\n",
    "train_set['SalePrice'] = train_set['SalePrice'].apply(np.log1p)\n",
    "\n",
    "y_train = train_set['SalePrice'].values\n",
    "n_train = train_set.shape[0]\n",
    "n_test = test_set.shape[0]\n",
    "x_train_df = train_set.drop('SalePrice', axis=1)\n",
    "x_test_df = test_set \n",
    "print(x_train_df.shape, x_test_df.shape)\n",
    "\n",
    "## missing data\n",
    "df = pd.concat((x_train_df, x_test_df)).reset_index(drop=True)\n",
    "\n",
    "## input missing value\n",
    "df['PoolQC'].fillna('None', inplace=True)\n",
    "df['MiscFeature'].fillna('None', inplace=True)\n",
    "df['Alley'].fillna('None', inplace=True)\n",
    "df['Fence'].fillna('None', inplace=True)\n",
    "df['FireplaceQu'].fillna('None', inplace=True)\n",
    "\n",
    "# Since the area of each street connected to the house, \n",
    "# probably most likely have a similary area to other houses in its neighboor\n",
    "df['LotFrontage'] = df.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))\n",
    "\n",
    "for col in ('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):\n",
    "    df[col] = df[col].fillna('None')\n",
    "\n",
    "for col in ('GarageYrBlt', 'GarageArea', 'GarageCars'):\n",
    "    df[col] = df[col].fillna(0)\n",
    "\n",
    "for col in ('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF',\n",
    "           'TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):\n",
    "    df[col] = df[col].fillna(0)\n",
    "\n",
    "for col in ('BsmtQual', 'BsmtCond', 'BsmtExposure',\n",
    "            'BsmtFinType1', 'BsmtFinType2'):\n",
    "    df[col] = df[col].fillna('None')\n",
    "\n",
    "df['MasVnrType'].fillna('None', inplace=True)\n",
    "df['MasVnrArea'].fillna(0, inplace=True)\n",
    "df['MSZoning'] = df['MSZoning'].fillna(df['MSZoning'].mode()[0])\n",
    "df = df.drop('Utilities', axis=1)\n",
    "df['Functional'].fillna('Typ', inplace=True)\n",
    "\n",
    "df['Electrical'].fillna(df['Electrical'].mode()[0], inplace=True)\n",
    "df['KitchenQual'].fillna(df['KitchenQual'].mode(), inplace=True)\n",
    "df['Exterior1st'] = df['Exterior1st'].fillna(df['Exterior1st'].mode()[0])\n",
    "df['Exterior2nd'] = df['Exterior2nd'].fillna(df['Exterior2nd'].mode()[0])\n",
    "df['SaleType'].fillna(df['SaleType'].mode()[0], inplace=True)\n",
    "df['MSSubClass'].fillna(df['MSSubClass'].mode()[0], inplace=True)\n",
    "\n",
    "df.isnull()[df.isnull() != 0].index\n",
    "\n",
    "df['MSSubClass'] = df['MSSubClass'].apply(str)\n",
    "df['OverallCond'] = df['OverallCond'].astype(str)\n",
    "df['YrSold'] = df['YrSold'].astype(str)\n",
    "df['MoSold'] = df['MoSold'].astype(str)\n",
    "\n",
    "train_set.drop('Id', axis=1, inplace=True)\n",
    "test_set.drop('Id', axis=1, inplace=True)\n",
    "df.drop('Id', axis=1, inplace=True)\n",
    "\n",
    "# label encode\n",
    "# 将特征稀疏的特征，在本列编码\n",
    "from sklearn.preprocessing import LabelEncoder \n",
    "\n",
    "cols = ('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',\n",
    "       'ExterQual', 'ExterCond', 'HeatingQC', 'PoolQC', 'KitchenQual', \n",
    "       'BsmtFinType1', 'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure',\n",
    "       'GarageFinish', 'LandSlope', 'LotShape', 'PavedDrive', 'Street', 'Alley',\n",
    "       'CentralAir', 'MSSubClass', 'OverallCond', 'YrSold', 'MoSold')\n",
    "\n",
    "for col in cols:\n",
    "    le = LabelEncoder()\n",
    "    le.fit(list(df[col].values))\n",
    "    df[col] = le.transform(list(df[col].values))\n",
    "\n",
    "# # dummies\n",
    "# df = pd.get_dummies(df)\n",
    "\n",
    "# x_train = df[:n_train].values\n",
    "# x_test = df[n_train:].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import ElasticNetCV, LassoCV, RidgeCV\n",
    "from sklearn.ensemble import GradientBoostingRegressor \n",
    "from sklearn.svm import SVR \n",
    "from sklearn.pipeline import make_pipeline \n",
    "from sklearn.preprocessing import RobustScaler \n",
    "from sklearn.model_selection import KFold, cross_val_score \n",
    "from sklearn.metrics import mean_squared_error \n",
    "from mlxtend.regressor import StackingCVRegressor \n",
    "from xgboost import XGBRegressor \n",
    "from lightgbm import LGBMRegressor \n",
    "\n",
    "\n",
    "kfold = KFold(n_splits=10, shuffle=True, random_state=42)\n",
    "\n",
    "def remsle(y, y_pred):\n",
    "    return np.sqrt(mean_squared_error(y, y_pred))\n",
    "\n",
    "def cv_rmse(model, X=x_train):\n",
    "    error = 'neg_mean_squared_error'\n",
    "    rmse = np.sqrt(-cross_val_score(model, X, y_train, scoring='neg_mean_squared_error', cv=kfolds))\n",
    "    return rmse \n",
    "\n",
    "# # ridge\n",
    "alpha_ridge = [14, 14.3, 14.6, 14.9, 15.2, 15.5]\n",
    "ridge = make_pipeline(RobustScaler(), RidgeCV(alphas=alpha_ridge, cv=kfold))\n",
    "# #lasso\n",
    "alpha_lasso = [0.0001, 0.0003, 0.001, 0.003, 0.01]\n",
    "lasso = make_pipeline(RobustScaler(), LassoCV(max_iter=1e5, \n",
    "                                                alphas=alpha_lasso, \n",
    "                                                random_state=42, \n",
    "                                                cv=kfolds))\n",
    "# # elaticnet\n",
    "alpha_ela = [0.0001, 0.0003, 0.001, 0.003, 0.01]\n",
    "e_l1ratio = [0.8, 0.85, 0.9, 0.95, 1]\n",
    "elasticnet = make_pipeline(RobustScaler(), ElasticNetCV(max_iter=1e5, \n",
    "                                                          alphas=alpha_ela, \n",
    "                                                          cv=kfolds, \n",
    "                                                          l1_ratio=e_l1ratio))\n",
    "# svr\n",
    "svr = make_pipeline(RobustScaler(), SVR(C=0.4, \n",
    "                                         epsilon=0.008,\n",
    "                                         gamma=0.0003))\n",
    "# # gradientboostregressor\n",
    "gbr = GradientBoostingRegressor(n_estimators=3000, \n",
    "                                learning_rate=0.05, \n",
    "                               max_depth=4,\n",
    "                               max_features='sqrt',\n",
    "                               min_samples_leaf=15,\n",
    "                               min_samples_split=10,\n",
    "                               loss='huber',\n",
    "                               random_state=42)\n",
    "\n",
    "# ligbtgbm\n",
    "lgb = LGBMRegressor(objective='regression',\n",
    "                   learning=0.01,\n",
    "                    e_estimators=5000,\n",
    "                    max_depth=4,\n",
    "                    num_leaves=3,\n",
    "                    feature_fraction=0.2,\n",
    "                    cat_smooth=0,\n",
    "                    bagging_fraction=0.9,\n",
    "                    bagging_fraq=3,\n",
    "                    verbose=-1\n",
    "                   )\n",
    "# xgboost\n",
    "xgb = XGBRegressor(learning_rate=0.01,\n",
    "                   n_estimators=3460,\n",
    "                   max_depth=3, \n",
    "                   min_child_weight=0,\n",
    "                   gamma=0, \n",
    "                   subsample=0.7,\n",
    "                   colsample_bytree=0.7,\n",
    "                   objective='reg:squarederror',\n",
    "                   nthread=-1,\n",
    "                   scale_pos_weight=1,\n",
    "                   seed=27,\n",
    "                   reg_alpha=0.00006)\n",
    "# stack model\n",
    "stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet, gbr, lgb, xgb),\n",
    "                               meta_regressor=lgb,\n",
    "                               use_features_in_secondary=True)\n",
    "\n",
    "# score = cv_rmse(ridge)\n",
    "# print('ridge 模型得分：{:.4f} ({:.4f})'.format(score.mean(), score.std()))\n",
    "\n",
    "# score = cv_rmse(lasso)\n",
    "# print('lasso 模型得分：{:.4f} ({:.4f})'.format(score.mean(), score.std()))\n",
    "\n",
    "# score = cv_rmse(elasticnet)\n",
    "# print('elasticnet 模型得分：{:.4f} ({:.4f})'.format(score.mean(), score.std()))\n",
    "\n",
    "# score = cv_rmse(svr)\n",
    "# print('svr 模型得分：{:.4f} ({:.4f})'.format(score.mean(), score.std()))\n",
    "\n",
    "# score = cv_rmse(gbr)\n",
    "# print('gradientboostregressor 模型得分：{:.4f} ({:.4f})'.format(score.mean(), score.std()))\n",
    "\n",
    "# score = cv_rmse(lgb)\n",
    "# print('lightgbm 模型得分：{:.4f} ({:.4f})'.format(score.mean(), score.std()))\n",
    "\n",
    "# score = cv_rmse(xgb)\n",
    "# print('xgboost 模型得分：{:.4f} ({:.4f})'.format(score.mean(), score.std()))\n",
    "\n",
    "# score = cv_rmse(stack_gen)\n",
    "# print('stacken model 模型得分：{:.4f} ({:.4f})'.format(score.mean(), score.std()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 用调好参数的模型拟合训练全部数据\n",
    "# ridge_ = ridge.fit(x_train, y_train)\n",
    "# lasso_ = lasso.fit(x_train, y_train)\n",
    "# elasticnet_ = elasticnet.fit(x_train, y_train)\n",
    "# svr_ = svr.fit(x_train, y_train)\n",
    "# gbr_ = gbr.fit(x_train, y_train)\n",
    "# lgb_ = lgb.fit(x_train, y_train)\n",
    "# xgb_ = xgb.fit(x_train, y_train)\n",
    "\n",
    "def blend_models_predict(x_test):\n",
    "    return ((0.1 * elasticnet.predict(x_test))+\n",
    "            (0.1 * ridge.predict(x_test))+\n",
    "            (0.1 * lasso.predict(x_test))+\n",
    "            (0.0 * svr.predict(x_test))+\n",
    "            (0.1 * gbr.predict(x_test))+\n",
    "            (0.1 * lgb.predict(x_test))+\n",
    "            (0.5 * xgb.predict(x_test))\n",
    "#             0.1 * elasticnet_.predict(x_test),\n",
    "           )\n",
    "\n",
    "# 融合后，训练模型对元数据拟合时均方根误差\n",
    "# print('训练数据拟合的均方根误差：', remsle(y_train, blend_models_predict(x_train)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([11.729377, 12.005816, 12.131836, ..., 11.967694, 11.656274,\n",
       "       12.327476], dtype=float32)"
      ]
     },
     "execution_count": 323,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = np.expm1(blend_models_predict(x_test))\n",
    "submission = pd.DataFrame()\n",
    "submission['Id'] = test_id\n",
    "submission['SalePrice'] = predictions\n",
    "submission.to_csv('submission_2020_03_01.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# svr 调参\n",
    "paramaters = {\n",
    "    'C':[0.01, 0.03, 0.1, 0.3, 1, 3, 10],\n",
    "    'gamma': [0.001, 0.003, 0.01, 0.03, 0.1],\n",
    "}\n",
    "\n",
    "svr = SVR(kernel='linear')\n",
    "gscv_svr = GridSearchCV(svr,\n",
    "                       param_grid = paramaters,\n",
    "                       cv=5,\n",
    "                       verbose=-1)\n",
    "gscv_svr.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "paramaters = {\n",
    "    'max_depth': [4],\n",
    "    'num_leaves': [3],\n",
    "    'feature_fraction': [0.2],\n",
    "    'cat_smmoth': [1],\n",
    "    'bagging_fraction': [0.9],\n",
    "    'bagging_fraq': [3]\n",
    "}\n",
    "\n",
    "gbm = lgb.LGBMRegressor(\n",
    "                objective = 'regression',\n",
    "                learning_rate = 0.01,\n",
    "                n_estimators = 5000,\n",
    "                verbose = -1\n",
    ")\n",
    "\n",
    "gscv = GridSearchCV(gbm,\n",
    "                   param_grid = paramaters,\n",
    "                   cv = 5,\n",
    "                   n_jobs = -1)\n",
    "\n",
    "gscv.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "linear_reg:-0.015115 (0.004384)\n",
      "knn:-0.048823 (0.005244)\n",
      "SVR:-0.156528 (0.020539)\n",
      "decision_tree:-0.037592 (0.007852)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAFOCAYAAACVGSr1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xu8XGV97/HPFwIoymVvbomBEJVUIxQQR7xhK3L3eEisiCBCsKEpbb20nlrBWBPRWKhHUVG0AapBJXKpQuSgECJKsKDuYEAQaRAFIgESsoEoggR/54/1bFzZzN772ZmVWXsm3/frNa+Ztdaz1vxmsjPfeZ51GUUEZmZmI9mi7gLMzKwzODDMzCyLA8PMzLI4MMzMLIsDw8zMsjgwzMwsiwPDDJB0gqRranrur0j6+Cba9rCvS9IbJK3cFM9t3ceBYZWS9A5JfZJ+K2mVpO9IOqjuukYSEV+PiMM35XNI+r6kfknbbMrnKRv8uiSFpL3a9fzWXRwYVhlJ7wc+A3wC2A2YBJwLTKuzrpFIGteG55gMvB4I4OhN/XzpOTf567LNiwPDKiFpB+AM4B8i4psR8buIeCoivh0RH0httpH0GUn3p9tnBr5tDwyNSPoXSQ+l3sl0SW+S9D+S1kr6UOn55kq6TNLFktZJulnSfqXlp0n6ZVr2c0lvKS07WdIPJZ0taS0wN827odQmJJ0qaUXqFXxBktKyLSV9StIaSb+S9O7UfrgP6JOAm4CvADNGeC//Jb3++yWdUu4VSNpB0oWSVku6R9KHJW2R87okXZ+e4pbUA3x76Tn/T+l9f1dp/lcknZt6ir9N2x+f/u36Jf1C0stL7T8o6Tfpfb9T0iHDvVbrLA4Mq8prgOcA3xqmzWzg1cD+wH7AgcCHS8vHp21MBD4CnAe8E3gFxbfzj0h6Uan9NOBSoBe4CLhc0lZp2S/TOjsAHwW+JmlCad1XAXcDuwLzhqj3zcArU63HAkek+X8DHJVexwHA9GFe84CTgK+n2xGSdmvWSNKRwPuBQ4G9gL8c1OSc9JpelJadBLyrtHzI1xURf5Ee7hcRz4+Ii9P0+LTNicBM4AuSekqrHkvx77Qz8CRwI3Bzmr4M+HSq/SXAu4FXRsR2FO/Xr4d5T6zDODCsKjsBayJi/TBtTgDOiIiHImI1xQf5iaXlTwHzIuIp4BsUH0ifjYh1EXE7cDuwb6n9soi4LLX/NEXYvBogIi6NiPsj4o/pg3EFRUANuD8izomI9RHx+yHqPTMiHomIe4HrKAICig/Qz0bEyojoB84c7o1J+3D2BC6JiGUUYfaOIZofC3w5Im6PiMfTezSwnS2BtwOnp/fk18Cn2PA9zHldgz1F8e/yVERcBfwWeElp+bciYllEPEHxheCJiLgwIp4GLgYGehhPA9sAL5O0VUT8OiJ+mVmDdQAHhlXlYWDnEYZlXgDcU5q+J817ZhvpQwhg4MPuwdLy3wPPL03fN/AgIv4IrBzYnqSTJC2X9IikR4B9KALoWesO44HS48dLz/2CQeuPtK0ZwDURsSZNX8TQw1LDbXtnYGue/R5OHEUtzTw8KOjLrxWe/W/Q9N8kIu4C/hGYCzwk6RuSyv++1uEcGFaVG4EnGH545n6Kb9oDJqV5G2uPgQdpHH934H5Je1IMZ70b2CkidgRuA1Rat5XLNK9Kz/WsOgaT9FyKXsNfSnpA0gPAPwH7lfe5ZG57DUVvYPB7+JvSdK2Xn46IiyJioEcVwFl11mPVcmBYJSLiUYr9Dl9IO6u3lbSVpKMk/XtqthD4sKRdJO2c2n+thad9haS/Sr2af6QYX78JeB7Fh9VqgLQTd58WnmewS4D3SZooaUfgg8O0nU4xVPMyiiGt/YGpwFKK/Q/Ntv0uSVMlbUvxHgGQel+XAPMkbZeC8f2M7j18kGL/R+UkvUTSG9OBDE9Q9D6eHmE16yAODKtMRHya4gPswxQf1vdRfMu/PDX5ONAH3Ar8jGLHaSsnrF1BMabfTzGO/1dpHP7nFGP7N1J8QP458MMWnmew84BrKF7HT4GrgPU0/3CcQbFP4t6IeGDgBnweOGHwEF5EfAf4HMU+k7vSa4AiDAHeA/yOYsf2DRTDW/85itrnAgvSUN2xo1gvxzYU+3PWUAzn7Qp8aNg1rKPIP6BknUjSXGCviHjnGKjlKOBLEbHniI1Hv+2pFMNp24xwQIHZJucehtkoSXpuOj9knKSJwByGP5x4tNt/i6St06GtZwHfdljYWODAMBs9URzu2k8xJHUHpX0NFfhbiiG9X1IMc/1dhds222gekjIzsyzuYZiZWRYHhpmZZXFgmJlZFgeGmZllcWCYmVkWB4aZmWVxYJiZWRYHhpmZZXFgmJlZFgeGmZllcWCYmVmWygJD0pGS7pR0l6TTmizfRtLFafmPJE0uLTs9zb9T0hG52zQzs/apJDDSj9N/ATiK4pfFjpf0skHNZgL9EbEXcDbppxtTu+OAvYEjgXMlbZm5TTMza5OqehgHAndFxN0R8QfgG8C0QW2mAQvS48uAQyQpzf9GRDwZEb+i+JWxAzO3aWZmbVJVYEyk+DnOASvTvKZt0o/BPArsNMy6OdtE0ixJfek2q8XXYWZmQxg3cpMsajJv8A9tDNVmqPnNwuxZP94REfOB+QA777xzNBqN/xi+VDMzK1u2bNmaiNhlpHZVBcZKYI/S9O7A/UO0WZl++H4HYO0I6460zQ1MnjyZvr6+URdvZrY5k3RPTruqhqR+AkyR9EJJW1PsxF40qM0iYEZ6fAzwvSh+7m8RcFw6iuqFwBTgx5nbNDOzNqmkhxER6yW9G7ga2BL4z4i4XdIZQF9ELAIuAL4q6S6KnsVxad3bJV0C/BxYD/xDRDwN0GybVdRrZmaj11W/6d1oNMJDUmZmoyNpWUQ0RmrnM73NzCyLA8PMzLI4MMzMLIsDw8zMsjgwzMwsS1Un7llSXB6rGt10BJuZdT4HRsVyPuQlOQzMrOM4MDL19vbS399f2faq6on09PSwdu3aSrZlZjYcB0am/v7+MdkrqHIIzMxsON7pbWZmWRwYZmaWxUNSmWLO9jB3h7rLeJaYs33dJZjZZsKBkUkffazuEprq6elh7dy6qzCzzYEDI1OVO7x9WK2ZdSLvwzAzsywODDMzy+LAMDOzLN6HUbHcE+ly2nk/h5mNJS31MCT1SlosaUW67xmi3YzUZoWkGWnetpL+n6RfSLpd0pml9idLWi1pebqd0kqd7RQRQ94uuugi9t57b7bYYgv23ntvLrroomHbm5mNJa0OSZ0GLImIKcCSNL0BSb3AHOBVwIHAnFKw/N+IeCnwcuB1ko4qrXpxROyfbue3WGftFi5cyOzZsznnnHN44oknOOecc5g9ezYLFy6suzQz2wiSKr11glYDYxqwID1eAExv0uYIYHFErI2IfmAxcGREPB4R1wFExB+Am4HdW6xnzJo3bx4XXHABBx98MFtttRUHH3wwF1xwAfPmzau7NDPbCMONDpRHCXLadcqIQquBsVtErAJI97s2aTMRuK80vTLNe4akHYH/TdFLGfBWSbdKukzSHi3WWbs77riDgw46aIN5Bx10EHfccUdNFZmZjc6IgSHpWkm3NblNy3yOZn2tZ+JU0jhgIfC5iLg7zf42MDki9gWu5U+9mGb1zZLUJ6lv9erVmSW139SpU7nhhhs2mHfDDTcwderUmioyMxudEQMjIg6NiH2a3K4AHpQ0ASDdP9RkEyuBcg9hd+D+0vR8YEVEfKb0nA9HxJNp8jzgFcPUNz8iGhHR2GWXXUZ6ObWZPXs2M2fO5LrrruOpp57iuuuuY+bMmcyePbvu0szMsrR6WO0iYAZwZrq/okmbq4FPlHZ0Hw6cDiDp48AOwAZHQUmaMDDUBRwNdPy4zfHHHw/Ae97zHu644w6mTp3KvHnznplvZjbWqZWdLZJ2Ai4BJgH3Am+LiLWSGsCpEXFKavfXwIfSavMi4suSdqfYt/ELYKA38fmIOF/Sv1EExXpgLfB3EfGLkeppNBrR19e30a/HzKxKnXLdOEnLIqIxYrtOeDG5HBhmNpZ0W2D40iBmZpbFlwYxMyvp7e2lv7+/su1VdVJeT08Pa9eurWRbG8uBYWZW0t/fPyaHkcbC2eAekjIzsywODDMzy+LAMDOzLA4MMzPL4sAwM7MsDgwzM8viwDAzsyw+D8PMrCTmbA9zd6i7jGeJOdvXXYIDw8auqk9UGosnY9nYo48+Nib/ViQRc+utwYFhY1buf9pOucCbWafzPgwzM8viwDAzsywekjIzG2QsXOhvsJ6enpEbbWIODDOzkir3h3Xb/jUHhtXCvzlg1nla3ochqVfSYkkr0n3TfpOkGanNCkkzSvO/L+lOScvTbdc0fxtJF0u6S9KPJE1utVYbOwZ+c2Cs3aoMMbNuU8VO79OAJRExBViSpjcgqReYA7wKOBCYMyhYToiI/dPtoTRvJtAfEXsBZwNnVVCrmZltpCoCYxqwID1eAExv0uYIYHFErI2IfmAxcOQotnsZcIjG4p4oM7PNRBX7MHaLiFUAEbFqYEhpkInAfaXplWnegC9Lehr4L+DjUewlemadiFgv6VFgJ2BNBTVbzXz5Bet0ud9fc9t1ws7xrMCQdC0wvsmi2ZnP0+wdG3h3ToiI30jajiIwTgQuHGGdcm2zgFkAkyZNyizH6ubLL1inG4t/v5taVmBExKFDLZP0oKQJqXcxAXioSbOVwBtK07sD30/b/k26XyfpIop9HBemdfYAVkoaB+wAPOvwlYiYD8wHaDQam9+/oJlZm1SxD2MRMHDU0wzgiiZtrgYOl9STdnYfDlwtaZyknQEkbQW8GbityXaPAb4Xm2Okm5mNEVXswzgTuETSTOBe4G0AkhrAqRFxSkSslfQx4CdpnTPSvOdRBMdWwJbAtcB5qc0FwFcl3UXRsziuglrNzGwjqZu+tDcajejr66u7DMswVg9484l7tjmStCwiGiO185neVgtffsGs8/hqtWZmlsU9DBuzRjNsldPWvRCz1jgwbMzyB7zZ2OIhKTMzy+LAMDOzLA4MMzPL4sAwM7MsDgwzM8viwDAzsywODDMzy+LAMDOzLA4MMzPL4sAwM7MsDgwzM8viwDAzsywODDMzy+LAMDOzLA4MMzPL0lJgSOqVtFjSinTfM0S7GanNCkkz0rztJC0v3dZI+kxadrKk1aVlp7RSp5mZta7VHsZpwJKImAIsSdMbkNQLzAFeBRwIzJHUExHrImL/gRtwD/DN0qoXl5af32KdZmbWolYDYxqwID1eAExv0uYIYHFErI2IfmAxcGS5gaQpwK7A0hbrMTOzTaTVwNgtIlYBpPtdm7SZCNxXml6Z5pUdT9GjKP8m51sl3SrpMkl7DFWApFmS+iT1rV69euNehZmZjWjEwJB0raTbmtymZT6Hmswb/GPNxwELS9PfBiZHxL7AtfypF/PsDUXMj4hGRDR22WWXzJLMzGy0xo3UICIOHWqZpAclTYiIVZImAA81abYSeENpenfg+6Vt7AeMi4hlped8uNT+POCskeo0M7NNq9UhqUXAjPR4BnBFkzZXA4dL6klHUR2e5g04ng17F6TwGXA0cEeLdZqZWYtG7GGM4EzgEkkzgXuBtwFIagCnRsQpEbFW0seAn6R1zoiItaVtHAu8adB23yvpaGA9sBY4ucU6zcysRdpwP3NnazQa0dfXV3cZZmYdRdKyiGiM1M5nepuZWRYHhpmZZXFgmJlZFgeGmZllcWCYmVkWB4aZmWVxYJiZWRYHhpmZZXFgmJlZFgeGmZllcWCYmVkWB4aZmWVxYJiZWRYHhpmZZXFgmJlZFgeGmZllcWCYmVkWB4aZmWVpOTAk9UpaLGlFuu8Zot13JT0i6cpB818o6Udp/YslbZ3mb5Om70rLJ7daq5mZbbwqehinAUsiYgqwJE0380ngxCbzzwLOTuv3AzPT/JlAf0TsBZyd2pmZWU2qCIxpwIL0eAEwvVmjiFgCrCvPkyTgjcBlTdYvb/cy4JDU3szMalBFYOwWEasA0v2uo1h3J+CRiFifplcCE9PjicB9abvrgUdTezMzq8G4nEaSrgXGN1k0u8Xnb9ZjiIxlf9qANAuYBTBp0qQWyzEzs6FkBUZEHDrUMkkPSpoQEaskTQAeGsXzrwF2lDQu9SJ2B+5Py1YCewArJY0DdgDWNqltPjAfoNFoPCtQzMysGlUMSS0CZqTHM4ArcleMiACuA45psn55u8cA30vtzcysBlUExpnAYZJWAIelaSQ1JJ0/0EjSUuBSip3XKyUdkRZ9EHi/pLso9lFckOZfAOyU5r+foY++MjOzNlA3fWlvNBrR19dXdxlmZh1F0rKIaIzUzmd6m5lZFgeGmZllcWCYmVkWB4aZmWVxYJiZWRYHhpmZZXFgmJlZFgeGmZllcWCYmVkWB4aZmWVxYJiZWRYHhpmZZXFgmJlZFgeGmZllcWCYmVkWB4aZmWVxYJiZWRYHhpmZZWkpMCT1SlosaUW67xmi3XclPSLpykHzvy7pTkm3SfpPSVul+W+Q9Kik5en2kVbqNDOz1rXawzgNWBIRU4AlabqZTwInNpn/deClwJ8DzwVOKS1bGhH7p9sZLdZpZmYtajUwpgEL0uMFwPRmjSJiCbCuyfyrIgF+DOzeYj1mZraJtBoYu0XEKoB0v+vGbCQNRZ0IfLc0+zWSbpH0HUl7t1inmZm1aNxIDSRdC4xvsmh2hXWcC1wfEUvT9M3AnhHxW0lvAi4HpgxR3yxgFsCkSZMqLMnMzMpGDIyIOHSoZZIelDQhIlZJmgA8NNoCJM0BdgH+tvScj5UeXyXpXEk7R8SaJvXNB+YDNBqNGO3zm5lZnlaHpBYBM9LjGcAVo1lZ0inAEcDxEfHH0vzxkpQeH5jqfLjFWs3MrAWtBsaZwGGSVgCHpWkkNSSdP9BI0lLgUuAQSSslHZEWfQnYDbhx0OGzxwC3SboF+BxwXNoxbmZmNVE3fQ43Go3o6+uruwwzs44iaVlENEZqN+I+DDPrDmmUtzLd9GXT8jgwzDYTOR/wkhwENiRfS8rMzLI4MMzMLIsDw8zMsjgwzMwsiwPDzMyyODDMzCyLA8PMzLI4MMzMLIsDw8zMsjgwzLpAb28vklq+AZVsRxK9vb01vytWNV8axKwL9Pf3j7lLelR97Sqrn3sYZmaWxYFhZmZZHBhmZpbFgWFmZlkcGGZmlqWlwJDUK2mxpBXpvmeIdt+V9IikKwfN/4qkX6Xf814uaf80X5I+J+kuSbdKOqCVOs3MrHWt9jBOA5ZExBRgSZpu5pPAiUMs+0BE7J9uy9O8o4Ap6TYL+GKLdZqZWYtaDYxpwIL0eAEwvVmjiFgCrBvldi+Mwk3AjpImtFSpmZm1pNXA2C0iVgGk+103Yhvz0rDT2ZK2SfMmAveV2qxM88zMrCYjnukt6VpgfJNFsyt4/tOBB4CtgfnAB4EzgGaniDY9jVXSLIphKyZNmlRBSWZm1syIgRERhw61TNKDkiZExKo0ZPTQaJ58oHcCPCnpy8A/p+mVwB6lprsD9w+xjfkUYUOj0Rhb10YwM+sirQ5JLQJmpMczgCtGs/LAfgkVF52ZDtxW2u5J6WipVwOPlsLFzMxq0OrFB88ELpE0E7gXeBuApAZwakSckqaXAi8Fni9pJTAzIq4Gvi5pF4ohqOXAqWm7VwFvAu4CHgfe1WKdZmbWIo21K1y2otFoRF9fX91lmLXf3B3qrqC5uY/WXYFlkLQsIhojtfPlzc26gD762Ji8vHnMrbsKq5IvDWJmZlkcGGZmlsWBYWZmWRwYZmaWxYFhZmZZHBhmZpbFgWFmZlkcGGZmlsWBYWZmWRwYZmaWxYFhZmZZHBhmZpbFgWFmZlkcGGZmlsWBYWZmWRwYZmaWxYFhZmZZHBhmZpalpcCQ1CtpsaQV6b5niHbflfSIpCsHzV8qaXm63S/p8jT/DZIeLS37SCt1mplZ61rtYZwGLImIKcCSNN3MJ4ETB8+MiNdHxP4RsT9wI/DN0uKlA8si4owW6zQzsxa1GhjTgAXp8QJgerNGEbEEWDfURiRtB7wRuLzFeszMbBNpNTB2i4hVAOl+143czlsoeiqPlea9RtItkr4jae8W6zQzsxaNG6mBpGuB8U0Wza6wjuOB80vTNwN7RsRvJb2JoucxZYj6ZgGzACZNmlRhSWadRVLdJWygp6fpLk3rYCMGRkQcOtQySQ9KmhARqyRNAB4abQGSdgIOpOhlDDznY6XHV0k6V9LOEbGmSX3zgfkAjUYjRvv8Zt0gopo/fUmVbcu6T6tDUouAGenxDOCKjdjG24ArI+KJgRmSxit9XZJ0YKrz4RZrNTOzFrQaGGcCh0laARyWppHUkPTMEJOkpcClwCGSVko6orSN44CFg7Z7DHCbpFuAzwHHhb/2mJnVSt30OdxoNKKvr6/uMsw6loekNk+SlkVEY6R2PtPbzMyyODDMzCyLA8PMzLI4MMzMLIsDw8zMsjgwzMwsiwPDzMyyODDMzCyLA8PMzLI4MMzMLIsDw8zMsox4eXMz6w65v5eR287XnNr8ODDMNhP+gLdWeUjKzMyyODDMzCyLA8PMzLI4MMzMLIsDw8zMsjgwzMwsiwPDzMyyODDMzCyLuulkHkmrgXvqriPDzsCauovoIn4/q+P3slqd8n7uGRG7jNSoqwKjU0jqi4hG3XV0C7+f1fF7Wa1uez89JGVmZlkcGGZmlsWBUY/5dRfQZfx+VsfvZbW66v30PgwzM8viHoaZmWVxYJiZWRYHhpmZZXFgmNkGJO0oaXbddXQ6Sc+ru4aq+Sda20jSAU1mPwrcExHr211PJ5O0C/A3wGRKf8cR8dd11dRpJO0B/CvwAuBy4CLgY8CJwMIaS+tokl4LnA88H5gkaT/gbyPi7+utrHUOjPY6FzgAuBUQsE96vJOkUyPimjqL6zBXAEuBa4Gna66lU10I/AD4L+BI4CbgdmDfiHigzsI63NnAEcAigIi4RdJf1FtSNRwY7fVrYGZE3A4g6WXAByi+1X0TcGDk2zYiPlh3ER2uNyLmpsdXS3oQeGVEPFljTV0hIu6TVJ7VFV9qvA+jvV46EBYAEfFz4OURcXeNNXWqKyW9qe4iOp2kHkm9knqBB4BtS9O2ce5Lw1IhaWtJ/wzcUXdRVfCJe20k6WJgLfCNNOvtFFezPBG4ISJeWVdtnUbSOuB5wJPAUxRDfBER29daWAeR9GtgqA+AiIgXtbGcriFpZ+CzwKEUf5fXAO+LiIdrLawCDow2kvRc4O+Bgyj+kG6g2K/xBMUQy29rLM/MbFgOjDZLoTEpIu6su5ZOJ2kisCcbHiV1fX0VdRZJPwe+BnzDw6LVkfRnwBeB3SJiH0n7AkdHxMdrLq1lDow2knQ08Elg64h4oaT9gTMi4uiaS+s4ks6iGNL7OX/aoRh+L/Olwz2PA46l+JGfhcAlEXF/rYV1OEk/oDiY5T8i4uVp3m0RsU+9lbXOR0m11xzgQOD7ABGxXNLkGuvpZNOBl/iIno0XEbcAtwCnS3o1RQDfJOkuYGFEnFdrgZ1r24j48aCjpLriPCsfJdVe6yPi0bqL6BJ3A1vVXUS3iIibIuKfgJOAHuDzNZfUydZIejHpgAJJxwCr6i2pGu5htNdtkt4BbClpCvBe4L9rrqlTPQ4sl7SE4kgpACLivfWV1JkkvRI4HngrxblC84FL66ypw/0DxXv4Ukm/AX4FnFBvSdVwYLTXe4DZFB9wFwFXAx2/I6wmN5LOpC3xIbWjIOkTFPsvHqE41Pt1EbGy3qo6m6QtgEZEHJquJbVFRKyru66qeKd3m0jaEjgzIj5Qdy3dQNLNwIyI+FmaPh74x4h4Vb2VdQ5JV1H8TV6fpk+i6GXcA8yNiLV11tepJF0fEV1xKZDBvA+jTSLiaeAVddfRRY4BFkiaKulvKM5vObzmmjrNeOA2gHStozMpri/1KF3206JttljSP0vaY+Cs+W45c949jDaS9ClgCsX48O8G5kfEN2srqoOl490vB+4DpkfE72suqaNIWh4R+6fHXwBWD1xbqrzMRkfSr5rM7ooz570Po716gYeBN5bmBcWFBy2DpJ+x4eUseoEtgR9JIiL2raeyjjRO0rh0af1DgFnlZTXV1A2mRsQT5RmSnlNXMVXyH0UbRcS7hlsu6fSI+Ld21dOh3lx3AV1kIfADSWuA31NcLh5Je1EMS9nG+W+KnzEYaV7H8ZDUGCLp5ojo+D8q6xzphL0JwDUR8bs078+A50fEzbUW12EkjQcmUlxu5R0U14uD4ui9L0XES+uqrSruYYwtGrmJWXUi4qYm8/6njlq6wBHAycDuwKf40//nx4AP1VRTpdzDGEPcwzDrfJLeGhH/NczyGRGxoJ01VcWH1Y4t7mGYdbjhwiJ5X1sK2QQcGG0iaUtJ/zRCM1+Owaz7dewXQwdGm6QT96aN0OYTbSrHzOrTsfsBvNO7vX4o6fPAxWx44p6PRjHbfHRsD8OB0V6vTfdnlOYFG57IZ2bd7Yd1F7CxfJSUmVmFJG1DcRHHyWz488FnDLVOp3APo80k/S9gb+CZSwV0wx+SmT3jCooz5ZdR+q2WbuDAaCNJXwK2BQ4Gzqe44uqPay3KzKq2e0QcWXcRm4KPkmqv10bESUB/RHwUeA2wR801mVm1/lvSn9ddxKbgHkZ7DVx++3FJL6C4cu0La6zHzKp3EHByusz5kxRHRUU3XEnZgdFeV0raEfgkcDPFEVLn11uSmVXsqLoL2FR8lFRN0pEUz4kIX0barMtI2g94fZpcGhG31FlPVbwPo40kbSvpXyWdFxFPArtK8u87mHURSe8Dvg7smm5fk/SeequqhnsYbSTpYopD7U6KiH0kPRe40T+FadY9JN0KvKb0+yLPo/h/3vH7MNzDaK8XR8S/A08BpN+g7tjLBJhZUwKeLk0/TZf8P/dO7/b6Q+pVBICkF9NlJ/aYGV+m+I35b6Xp6cAFNdZTGQ9JtZGkw4APAy8DrgFeB5wcEd+vsy4zq5akAygOrxVwfUT8tOaSKuHAaDNJOwGvpvhDuiki1tRckplVQNL2EfGYpN5myyNibbtrqpoDo80kTQRl615FAAACmklEQVT2ZMOLkl1fX0VmVgVJV0bEm9MJe+UP1oET915UU2mVcWC0kaSzgLcDtwN/TLMjIo6uryozszwOjDaSdCewbzoHw8y6kKTXAcsj4neS3gkcAHwmIu6tubSW+bDa9rob2KruIsxsk/oixfXi9gP+BbgH+Gq9JVXDh9W21+PAcklLKB1OGxHvra8kM6vY+ogISdOAz0bEBZJm1F1UFRwY7bUo3cyse62TdDrwTuAvJG1Jl4wseB+GmVmFJI0H3gH8JCKWSpoEvCEiLqy5tJY5MNpA0iURcaykn7Hh4XYAdMM1Zsys+zkw2kDShIhYJWnPZssj4p5212Rm1ZJ0Q0QcJGkdzc/D2L6m0irjwDAzsyze6d0GTb5xPLOILvnmYWYFSa8Gbo+IdWn6+cDeEfGjeitrnXsYZmYVkvRT4IBIH66StgD6IuKAeitrnU/cMzOrlqL0TTwi/kiXjOY4MMzMqnW3pPdK2ird3kdxlYeO58AwM6vWqcBrgd8AK4FXAbNqragi3odhZmZZ3MMwM6uQpD+TtETSbWl6X0kfrruuKjgwzMyqdR5wOvAUQETcChxXa0UVcWCYmVVr24j48aB562uppGIODDOzaq2R9GLSybqSjgFW1VtSNbzT28ysQpJeBMynOFKqH/gVcEI3XDPOgWFmVgFJ7x8067kUozi/A4iIT7e9qIp1xdmHZmZjwHbp/iXAK4ErKK4XdyJwfV1FVck9DDOzCkm6Bnhr6eKD2wGXRsSR9VbWOu/0NjOr1iTgD6XpPwCT6ymlWh6SMjOr1leBH0v6FsWRUm8BFtRbUjU8JGVmVjFJBwCvT5PXR8RP66ynKg4MMzPL4n0YZmaWxYFhZmZZHBhmZpbFgWFmZlkcGGZmluX/A4YFqRV6N/LaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.svm import SVR \n",
    "from sklearn.tree import DecisionTreeRegressor \n",
    "\n",
    "from sklearn.model_selection import KFold \n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "models = []\n",
    "models.append(('linear_reg', LinearRegression()))\n",
    "models.append(('knn', KNeighborsRegressor()))\n",
    "models.append(('SVR', SVR()))\n",
    "models.append(('decision_tree', DecisionTreeRegressor()))\n",
    "\n",
    "names = []\n",
    "predictions = []\n",
    "error = 'neg_mean_squared_error'\n",
    "for name, model in models:\n",
    "    fold = KFold(n_splits=10, random_state=0)\n",
    "    result = cross_val_score(model, x_train, y_train, cv=fold, scoring=error)\n",
    "    predictions.append(result)\n",
    "    names.append(name)\n",
    "    msg = '%s:%f (%f)' % (name, result.mean(), result.std())\n",
    "    print(msg)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create pipeline with Standardization scale and models \n",
    "\n",
    "### Standardize the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scaler_lg: -42299828435270451200.000000 (111249075729436999680.000000)\n",
      "scale_KNN: -0.039544 (0.008133)\n",
      "scale_SVR: -0.031837 (0.007543)\n",
      "scale_decision: -0.038644 (0.008253)\n",
      "[array([-2.35386430e+16, -2.10108418e+17, -4.28381611e+18, -3.71453484e+15,\n",
      "       -2.35728473e+18, -5.50305990e+17, -3.76904730e+18, -1.09732242e+18,\n",
      "       -3.74582364e+20, -3.61207825e+19]), array([-0.03775905, -0.04253397, -0.04238442, -0.04563515, -0.05259116,\n",
      "       -0.02740558, -0.04524507, -0.02692457, -0.03091265, -0.04404923]), array([-0.03696491, -0.02866518, -0.03998731, -0.03476248, -0.04472446,\n",
      "       -0.02519728, -0.03448589, -0.01978869, -0.02207881, -0.03171623]), array([-0.04559179, -0.04219368, -0.03927766, -0.0459086 , -0.04406778,\n",
      "       -0.02143378, -0.03340501, -0.04624315, -0.02654237, -0.04178102])]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEVCAYAAADpbDJPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGJxJREFUeJzt3X+UXWV97/H3xyEQBaRBFAghRoXSSUcI7VmxCr0SiDQiDdpelUjbaIem1Gtuf7iq0rlVQHMXXtZtuy7apWmhoMIgtjeLVIIQYFI6Ksqk5UdCooTIjzhgJiQSFINJ+N4/9jPcw3jOzJnZJ3PO5Pm81jpr9o9n9vOcPcn+nP08++ytiMDMzPLzilY3wMzMWsMBYGaWKQeAmVmmHABmZplyAJiZZcoBYGaWKQeA1STpOkmfOUDbvkjSHaOsP0vStgNR98FK0mxJP5HUMUa5Ufe95cUBkDlJ6yTtknTYZNUZETdExLlVbQhJJ01W/aORNF/SGkk/lrRT0nclfajV7RpLRDwREUdExP4xyr1s31veHAAZkzQH+E0ggMWTVOchk1HPREh6K3A38G/AScBrgD8B3tnKdo2lnfeptTcHQN7+ALgXuA5YOlpBSR+T9JSkQUkXV39ql3SUpC9JGpL0uKT/IekVad0HJX1T0t9K2glclpb1p/X3pCoeSF0Y76+q86OStqd6P1S1/DpJfy/ptvQ735R0nKS/S2czmyWdXlX+45J+KOk5Sd+TdE6dt3kVcH1EfDYidkRhfUS8r2pbfyRpSzo7WC1pZtW6kPRhSY+kuj4t6U2Svi1pt6SbJR2ayp4laZukv5K0Q9Jjki6q2ta7JP1n+r0nJV1WtW5Oqqtb0hPA3VXLDqna71tTO34wvO3qfZ/m3ybpPknPpp9vq1q3Lr2Hb6bt3CHpmLRuuqSvSHomnS3dJ+nY0f4NWRuKCL8yfQFbgA8Dvw7sBY6tWncd8Jk0vQh4GvhV4FXAlynOGk5K678E3AIcCcwBvg90p3UfBPYBy4FDgFemZf1Vdb20rTR/VvqdK4BpwHnA88CMqrbtSO2eTvGp/QcUgdYBfAboS2VPAZ4EZqb5OcCbauyLVwH7gQWj7K+zU72/BhwGXA3cM+J9rAZenfbVC8BdwBuBo4CHgaUj3uPfpG29HfgpcErV+jdTfEg7FfgR8O6q9xBpvx+e9unwskPSst1V2zoe+NWqv0d/mj4a2AX8fvq9JWn+NWn9OuBR4JdTHeuAK9O6Pwb+Ne23jvS3eHWr/037Nb6XzwAyJelM4PXAzRGxnuI/+gfqFH8f8E8RsTEingcur9pOB/B+4NKIeC4iHgP+N8VBZdhgRFwdEfsi4mcNNnEvcEVE7I2INcBPKA7mw1ZF8el8D7AK2BMRX4qiD/yrwPAZwH6KA+xcSdMi4rGIeLRGfTMoDrZPjdKmi4BrI+I/IuIF4FLgrakrbdhnI2J3RGwENgB3RMTWiHgWuK2qXcP+OiJeiIh/A26l2NdExLqIeCgiXoyIB4FeipCodllE/LTOPn0R6JL0yoh4KrVnpHcBj0TEl9PfphfYDPx2VZl/iojvpzpuBual5XspushOioj96W+xu96Os/bkAMjXUoqD0440fyP1u4FmUnyKHlY9fQxwKPB41bLHgRPqlG/UMxGxr2r+eeCIqvkfVU3/rMb8EQARsQX4M+AyYLukm6q7barsojhoHj9Km2ZS9T4j4ifAM7z8vTbUruE6I+KnVfOPpzqQ9BZJfalb7VngEop9Xa3mfk3bfH/6nack3SrpV8Z6P1VtqH4/T1dNV/8NvgzcDtyUugX/l6Rptdpj7csBkCFJr6T4pPl2SU9Lehr4c+A0SafV+JWngFlV8ydWTe+g+DT4+qpls4EfVs239JazEXFjRAyf8QTw2Rplnge+DfzuKJsapOp9Sjqc4lPwD+v+xuhmpG0Mm53qgCKQVwMnRsRRwBcAjWx2vQ1HxO0R8Q6KQNsM/EONYi97P1VtGPP9pDOzyyNiLvA24HyKLjibQhwAeXo3RdfIXIpT+nlAJ/Dv1P5PfDPwIUmdkl4FfHJ4RepyuRlYIelISa8H/gL4yjja8yOKfvKmk3SKpLNVXOa6h+JTeL1LJT8GfFDSX0p6Tfr90yTdlNbfSLEf5qXt/U/gO6nba6Iul3SopN+kOIh+LS0/EtgZEXskzad+99wvkHSspMUpXF6g6D6r9Z7XAL8s6QOSDkkD8HOBrzdQxwJJb05dgLspPgSMegmqtR8HQJ6WUvTtPhERTw+/gM8BF2nEZYURcRvwf4A+ioHjb6dVL6SfyykGMLcC/RQHymvH0Z7LgOvT1STvG6vwOB0GXElxpvI08Drgr2oVjIhvUQz0ng1sVXHV0kqKAyURcRfw18C/UJwVvQm4sETbnqboehoEbgAuiYjNad2HgSskPUcRuDePY7uvAD6atruTYuzgwyMLRcQzFKHzUYqurI8B51d1C47mOOCfKQ7+mygunR1P6FsbUIQfCGPjI6mTYoDzsBH99NYgSWcBX4mIWWOVNTtQfAZgDZH0ntRVMYOiD/1fffA3m9ocANaoPwaGKC4X3U/xDVkzm8LcBWRmlimfAZiZZcoBYGaWKQeAmVmmHABmZplq+wCQdK2KWwJvaKDsX0h6WNKDku5K30odXrc03ab3EUmj3vrYzCwHbX8VkKT/QvFV9i9FRNcYZRdQfDX/eUl/ApwVEe+XdDQwAFQo7p+yHvj1iNh1gJtvZta22v4MICLuofg6+0vSQza+IWm9pH8fvtNhRPSlm3pB8aCT4W9Z/hawNiJ2poP+Wop73JuZZWuqPkpuJcV9Ux6R9Bbg7ynu31Ktm+L+61Dc3rb61rnbePktb83MsjPlAkDSERS3n/2a9NLdcQ8bUeb3KLp7hh+gMfI2utDiWxSbmbXalAsAim6rH0fEvForJS0EeoC3p6c2QfGJ/6yqYrMoHm9nZpatth8DGCk9du4Hkt4LoMJpafp04IvA4ojYXvVrtwPnSpqRbmZ2blpmZpattg8ASb0U958/RdI2Sd0Uz2btlvQAsBG4IBW/iuKRdV+TdL+k1QARsRP4NHBfel2RlpmZZavtLwM1M7MDo+3PAMzM7MBo60HgY445JubMmdPqZpiZTRnr16/fERGvbaRsWwfAnDlzGBgYaHUzzMymDEmPN1rWXUBmZplyAJiZZcoBYGaWKQeAmVmmHABmZplqSgBIWiTpe5K2SPpEjfWHSfpqWv8dSXOaUa+ZmU1c6QCQ1AF8HngnMBdYImnuiGLdwK6IOAn4W+CzZes1M7NymnEGMB/YEhFbI+LnwE38/3vzDLsAuD5N/zNwjqru5WxmZpOvGV8Eq/WwlbfUKxMR+yQ9C7wG2DFyY5KWAcsAZs+e3YTm1Xb00Ueza1d7PRFyxowZ7Nw5Be9Rd9lRrW5BfZc92+oWjJ/3Z3O16/5sg33ZjABo5GErDT+QJSJWUjzxi0qlcsDuVLfzv+8HXn2gNj9B+1vdgIlpg3/IBxXvz+by/qyrGQGwDTixan4WMFinzDZJhwBHMeI5v5NNl+9uZfU1zZgxg52XtboVZpaLZgTAfcDJkt4A/BC4EPjAiDKrgaUU9/X/r8Dd0eL7UDdSfbOHKXzrbTNrJ6UDIPXpf4TiCVsdwLURsVHSFcBARKwGrgG+LGkLxSf/C8vWOxl8wDazg1lT7gYaEWuANSOWfbJqeg/w3mbUZWZmzeFvApuZZcoBYGaWKQeAmVmmHABmZplyAJiZZcoBYGaWKQeAmVmmHABmZplyAJiZZcoBYGaWKQfABPX29tLV1UVHRwddXV309va2uklmZuPSlHsB5aa3t5eenh6uueYazjzzTPr7++nu7gZgyZIlLW6dmVlj1M53vKxUKjEwMNDqZvyCrq4urr76ahYsWPDSsr6+PpYvX86GDRta2DIzy52k9RFRaaisA2D8Ojo62LNnD9OmTXtp2d69e5k+fTr790/Rp3qZ2UFhPAHgMYAJ6OzspL+//2XL+vv76ezsbFGLzMzGzwEwAT09PXR3d9PX18fevXvp6+uju7ubnp6eVjfNzKxhHgSegOGB3uXLl7Np0yY6OztZsWKFB4DNbEopNQYg6Wjgq8Ac4DHgfRGxq0a5/cBDafaJiFjcyPbbdQzAzKxdTeYYwCeAuyLiZOCuNF/LzyJiXno1dPA3M7MDq2wAXABcn6avB95dcntmZjZJygbAsRHxFED6+bo65aZLGpB0rySHhJlZGxhzEFjSncBxNVaN55KX2RExKOmNwN2SHoqIR+vUtwxYBjB79uxxVGFmZuMxZgBExMJ66yT9SNLxEfGUpOOB7XW2MZh+bpW0DjgdqBkAEbESWAnFIPCY78DMzCakbBfQamBpml4K3DKygKQZkg5L08cAZwAPl6zXzMxKKhsAVwLvkPQI8I40j6SKpH9MZTqBAUkPAH3AlRHhADAza7FSXwSLiGeAc2osHwAuTtPfAt5cph4zM2s+3wrCzCxTDgAzs0w5AMzMMuUAMDPLlAPAzCxTDgAzs0w5AMzMMuUAMDPLlAPAzCxTDgAzs0w5AMzMMuUAMDPLlAPAzCxTDgAzs0w5AMzMMuUAMDPLlAPAzCxTDgAzs0yVCgBJ75W0UdKLkiqjlFsk6XuStkj6RJk6zcysOcqeAWwAfge4p14BSR3A54F3AnOBJZLmlqzXzMxKKvtQ+E0AkkYrNh/YEhFbU9mbgAuAh8vUbWZm5UzGGMAJwJNV89vSspokLZM0IGlgaGjogDfOzCxXY54BSLoTOK7Gqp6IuKWBOmqdHkS9whGxElgJUKlU6pYzM7NyxgyAiFhYso5twIlV87OAwZLbNDOzkiajC+g+4GRJb5B0KHAhsHoS6jUzs1GUvQz0PZK2AW8FbpV0e1o+U9IagIjYB3wEuB3YBNwcERvLNdvMzMoqexXQKmBVjeWDwHlV82uANWXqMjOz5vI3gc3MMuUAMDPLlAPAzCxTDgAzs0w5AMzMMuUAMDPLlAPAzCxTDgAzs0w5AMzMMuUAMDPLlAPAzCxTDgAzs0w5AMzMMuUAMDPLlAPAzCxTDgAzs0w5AMzMMlX2kZDvlbRR0ouSKqOUe0zSQ5LulzRQpk4zM2uOUo+EBDYAvwN8sYGyCyJiR8n6zMysSco+E3gTgKTmtMbMzCbNZI0BBHCHpPWSlo1WUNIySQOSBoaGhiapeWZm+RnzDEDSncBxNVb1RMQtDdZzRkQMSnodsFbS5oi4p1bBiFgJrASoVCrR4PbNzGycxgyAiFhYtpKIGEw/t0taBcwHagaAmZlNjgPeBSTpcElHDk8D51IMHpuZWQuVvQz0PZK2AW8FbpV0e1o+U9KaVOxYoF/SA8B3gVsj4htl6jUzs/LKXgW0ClhVY/kgcF6a3gqcVqYeMzNrPn8T2MwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsU2UfCXmVpM2SHpS0StIv1Sm3SNL3JG2R9IkydZqZWXOUPQNYC3RFxKnA94FLRxaQ1AF8HngnMBdYImluyXrNzKykUgEQEXdExL40ey8wq0ax+cCWiNgaET8HbgIuKFOvmZmV18wxgD8Ebqux/ATgyar5bWmZmZm10CFjFZB0J3BcjVU9EXFLKtMD7ANuqLWJGstilPqWAcsAZs+ePVbzzMxsgsYMgIhYONp6SUuB84FzIqLWgX0bcGLV/CxgcJT6VgIrASqVSt2gMDOzcspeBbQI+DiwOCKer1PsPuBkSW+QdChwIbC6TL1mZlZe2TGAzwFHAmsl3S/pCwCSZkpaA5AGiT8C3A5sAm6OiI0l6zUzs5LG7AIaTUScVGf5IHBe1fwaYE2ZuszMrLn8TWAzs0w5AMzMMuUAMDPLlAPAzCxTDgAzs0w5AMzMMuUAMDPLlAPAzCxTDgAzs0w5AMzMMuUAMDPLlAPAzCxTDgAzs0w5AMzMMuUAMDPLlAPAzCxTDgAzs0w5AMzMMlXqkZCSrgJ+G/g58CjwoYj4cY1yjwHPAfuBfRFRKVOvmZmVV/YMYC3QFRGnAt8HLh2l7IKImOeDv5lZeygVABFxR0TsS7P3ArPKN8nMzCZDM8cA/hC4rc66AO6QtF7SstE2ImmZpAFJA0NDQ01snpmZVRtzDEDSncBxNVb1RMQtqUwPsA+4oc5mzoiIQUmvA9ZK2hwR99QqGBErgZUAlUolGngPZmY2AWMGQEQsHG29pKXA+cA5EVHzgB0Rg+nndkmrgPlAzQAwM7PJUaoLSNIi4OPA4oh4vk6ZwyUdOTwNnAtsKFOvmZmVV3YM4HPAkRTdOvdL+gKApJmS1qQyxwL9kh4AvgvcGhHfKFmvmZmVVOp7ABFxUp3lg8B5aXorcFqZeszMrPn8TWAzs0w5AMzMMuUAMDPLlAPAzCxTDgAzs0w5AMzMMuUAMDPLlAPAzCxTDgAzs0w5AMzMMuUAMDPLlAPAzCxTDgAzs0w5AMzMMuUAMDPLlAPAzCxTDgAzs0yVDgBJn5b0YHok5B2SZtYpt1TSI+m1tGy9ZmZWTjPOAK6KiFMjYh7wdeCTIwtIOhr4FPAWYD7wKUkzmlC3mZlNUOkAiIjdVbOHA1Gj2G8BayNiZ0TsAtYCi8rWbWZmE1fqofDDJK0A/gB4FlhQo8gJwJNV89vSMjMza5GGzgAk3SlpQ43XBQAR0RMRJwI3AB+ptYkay2qdKSBpmaQBSQNDQ0ONvg8zMxunhs4AImJhg9u7EbiVor+/2jbgrKr5WcC6OnWtBFYCVCqVmiFhZmblNeMqoJOrZhcDm2sUux04V9KMNPh7blpmZmYt0owxgCslnQK8CDwOXAIgqQJcEhEXR8ROSZ8G7ku/c0VE7GxC3WZmNkGKaN9elkqlEgMDA61uhpnZlCFpfURUGinrbwKbmWXKAWBmlikHgJlZphwAZmaZcgCYmWXKAWBmlikHgJlZphwAZmaZcgCYmWXKAWBmlikHgJlZphwAZmaZcgCYmWXKAWBmlikHgJlZphwAZmaZcgCYmWXKAWBmlqlSzwROz/m9gOJ5wNuBD0bEYI1y+4GH0uwTEbG4TL1mZlZe2TOAqyLi1IiYB3wd+GSdcj+LiHnp5YO/mVkbKBUAEbG7avZwoH2fMG9mZi9TegxA0gpJTwIXUf8MYLqkAUn3Snr3GNtblsoODA0NlW2emZnVoYjRP7RLuhM4rsaqnoi4parcpcD0iPhUjW3MjIhBSW8E7gbOiYhHx2pcpVKJgYGBsYqZmVkiaX1EVBopO+YgcEQsbLDeG4FbgV8IgOGB4YjYKmkdcDowZgCYmdmBU6oLSNLJVbOLgc01ysyQdFiaPgY4A3i4TL1mZlZeqctAgSslnUJxGejjwCUAkirAJRFxMdAJfFHSixSBc2VEOADMzFqsVABExO/WWT4AXJymvwW8uUw9ZmbWfP4msJlZphwAZmaZcgCYmWXKAWBmlikHwAT19vbS1dVFR0cHXV1d9Pb2trpJZmbjUvYy0Cz19vbS09PDNddcw5lnnkl/fz/d3d0ALFmypMWtMzNrzJi3gmildr0VRFdXF1dffTULFix4aVlfXx/Lly9nw4YNLWyZmeVuPLeCcABMQEdHB3v27GHatGkvLdu7dy/Tp09n//79LWyZmeVuPAHgMYAJ6OzspL+//2XL+vv76ezsbFGLzMzGzwEwAT09PXR3d9PX18fevXvp6+uju7ubnp6eVjfNzKxhHgSegOGB3uXLl7Np0yY6OztZsWKFB4DNbErxGICZ2UHEYwBmZjYmB4CZWaYcAGZmmXIAmJllygFgZpaptr4KSNIQxaMm29kxwI5WN+Ig4v3ZXN6fzTUV9ufrI+K1jRRs6wCYCiQNNHrJlY3N+7O5vD+b62Dbn+4CMjPLlAPAzCxTDoDyVra6AQcZ78/m8v5sroNqf3oMwMwsUz4DMDPLlANggiRdK2m7JD8CrAkknSipT9ImSRsl/Wmr2zRVSZou6buSHkj78vJWt+lgIKlD0n9K+nqr29IsDoCJuw5Y1OpGHET2AR+NiE7gN4D/Jmlui9s0Vb0AnB0RpwHzgEWSfqPFbToY/CmwqdWNaCYHwARFxD3Azla342AREU9FxH+k6eco/qOd0NpWTU1R+EmanZZeHuwrQdIs4F3AP7a6Lc3kALC2I2kOcDrwnda2ZOpK3RX3A9uBtRHhfVnO3wEfA15sdUOayQFgbUXSEcC/AH8WEbtb3Z6pKiL2R8Q8YBYwX1JXq9s0VUk6H9geEetb3ZZmcwBY25A0jeLgf0NE/N9Wt+dgEBE/Btbh8aoyzgAWS3oMuAk4W9JXWtuk5nAAWFuQJOAaYFNE/E2r2zOVSXqtpF9K068EFgKbW9uqqSsiLo2IWRExB7gQuDsifq/FzWoKB8AESeoFvg2cImmbpO5Wt2mKOwP4fYpPV/en13mtbtQUdTzQJ+lB4D6KMYCD5tJFax5/E9jMLFM+AzAzy5QDwMwsUw4AM7NMOQDMzDLlADAzy5QDwMwsUw4AM7NMOQDMzDL1/wDup3yuwd3g1AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler \n",
    "\n",
    "pipelines = []\n",
    "pipelines.append(('scaler_lg', Pipeline([('scaler', MinMaxScaler()), ('lg', LinearRegression())])))\n",
    "pipelines.append(('scale_KNN', Pipeline([('scaler', StandardScaler()), ('KNN', KNeighborsRegressor())])))\n",
    "pipelines.append(('scale_SVR', Pipeline([('scaler', StandardScaler()), ('SVR', SVR())])))\n",
    "pipelines.append(('scale_decision', Pipeline([('scaler', StandardScaler()), ('decision', DecisionTreeRegressor())])))\n",
    "\n",
    "predictions = []\n",
    "names = []\n",
    "for name, model in pipelines:\n",
    "    fold = KFold(n_splits=10, random_state=5)\n",
    "    result = cross_val_score(model, x_train, y_train, cv=fold, scoring=error)\n",
    "    predictions.append(result)\n",
    "    names.append(name)\n",
    "    msg = '%s: %f (%f)' % (name, result.mean(), result.std())\n",
    "    print(msg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.015294 useing {'C': 0.4, 'kernel': 'linear'}\n"
     ]
    }
   ],
   "source": [
    "# SVR tuning \n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "scaler = StandardScaler().fit(x_train)\n",
    "rescaledx = scaler.transform(x_train)\n",
    "\n",
    "param_grid = {\n",
    "    'kernel': ['linear', 'poly', 'rbf', 'sigmoid'],\n",
    "    'C': [0.2, 0.4, 0.6, 0.8, 1.0]\n",
    "}\n",
    "\n",
    "model = SVR()\n",
    "fold = KFold(n_splits=10, random_state=42)\n",
    "\n",
    "grid = GridSearchCV(estimator=model,\n",
    "                   param_grid=param_grid,\n",
    "                   scoring=error,\n",
    "                   cv=fold)\n",
    "grid_result = grid.fit(rescaledx, y_train)\n",
    "print('Best: %f useing %s' % (grid_result.best_score_, grid_result.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.038076 using {'criterion': 'friedman_mse'}\n"
     ]
    }
   ],
   "source": [
    "# DecisionTreeRegressor Tuning\n",
    "scaler = StandardScaler().fit(x_train)\n",
    "rescalex = scaler.transform(x_train)\n",
    "\n",
    "criterion = ['mse', 'friedman_mse', 'mae']\n",
    "param_grid = dict(criterion = criterion)\n",
    "\n",
    "model = DecisionTreeRegressor()\n",
    "fold = KFold(n_splits=10, random_state=42)\n",
    "grid = GridSearchCV(estimator=model,\n",
    "                   param_grid = param_grid,\n",
    "                   scoring = error,\n",
    "                   cv=fold)\n",
    "grid_result = grid.fit(rescaledx, y_train)\n",
    "\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scaledAB : -0.029019 (0.003039)\n",
      "scaledGBR : -0.015235 (0.004337)\n",
      "scaledRF : -0.020676 (0.003583)\n",
      "scaledETR : -0.019509 (0.004989)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEVCAYAAADzUNLBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGcVJREFUeJzt3X20XXdd5/H3hyRQSmmbQCmxaZpBWRqMgHoX1aGOLfSJx1YRpQuZdEytHZwCI7OGahhTKGEVlg/AQgcLcVEUr2BHp62OU9OSjhMVbcIUBgwaREoioU+5UAoW0853/jg75fRyb+4vOSf3nHvzfq211z1779/Z+7v3ufd8zv7tffZNVSFJ0lweN+oCJEkLg4EhSWpiYEiSmhgYkqQmBoYkqYmBIUlqYmBo3iU5O8neQ8z/YJK3zWdNC1GSLyQ59wife2mS7cOuqVv26iQPJllyiDaV5LuOxvp19BgYi1D3RvLP3R/tweG9o65rPqTndUk+neTrSfYm+YMk3zfq2kYhydXdm/Pz5mudVfXFqjqhqh7parg9yWXztX4dPUtHXYCOmpdV1a2jLmIE3g28BPhZ4C+AJcCPddP+7wjrOqQkS6vq4SEvM8BrgP3AeuBvhrn8WdY59O3Q+PAI4xhzsCsiya8kmUryj0leNG3+55N8rZv36r55P5NkV/e8W5Kc0Tevkrw2ye7uudck+c4kf5XkgSQfTfL4abX8UpL7uiOiVzOLJC9NcmeSryT5yyTPnqXdM4GfBy6pqo9V1Ter6htV9eGqurZrc1KSDyW5N8ldSd6c5HF92/4XSX69W9fnk/zrbvqeJPckWd+3vg8meV+Srd02/69p++Td3fMeSLIzyY/0zbs6yQ1JfjfJA8ClSR6X5Kok/5Dk/m6freh7zmu6mu9PsnGOlxrgR4DvAF4PvGr6/p+2785P8ndJvprkN7ttuayb97huP93V7YMPJTmpm7eme+03JPki8LG+aUuTbO7qeO8MR7rndr8vU0l+owu4I3kdXpzkb7vX4J+S/KeGfaMjUVUOi2wAvgCcO8u8S4ED9D6BLwH+PfAlIMCTgAeA7+7argS+t3t8MfA5YC29I9M3A3/Zt9wCbgJOBL4X+CZwG/AM4CTgb4H1XduzgYeBXwOeAPwo8PW+9X4QeFv3+AeAe4Azu3rXd9v3hBm27Qrgrjn2zYeAG4EnA2uAvwc29O2bh4F/163rbcAXgd/o6jwf+BpwQl+dXwP+TTf/3cD2vnX9NPCUbn+9EfgycFw37+rudbiY3ge3JwJvAD4OrOqW91vAZNf+WcCDfev6ta7WGV/n7jlbgI8Cy4D7gR+f9nuwvXv81O51//Gu1td3tV3Wzf+Z7rV/BnAC8IfA73Tz1nSv/Yfo/f48sW/a0q7N7QeXNe335Y+Bk4HVwL3AhUf4OuwDfqR7vBz4gVH/DS7WYeQFOByFF7X3hvog8JW+4We7eZcCn+tre3z3x/v07g/+K8ArgCdOW+af0r2xduOPA74BnNGNF/D8vvk7gTf1jf8q8K7u8dndG8KT+uZ/FPgv3eMP8q3A+K/ANdNq+TvgR2fY7o3Axw+xX5bQC7Jn9U37OeD2vn2zu2/e93XbdWrftPuB5/bV+ft9804AHgFOn2X9U8BzusdXA38+bf4u4IV94yvpvXEvBX552rqeBPwLs38wOJ5eCFzcjf8WcGPf/Ev5VmD8W+Cv+uYF2MO3AuM24LV987+7r6413T56Rt/8g9PmCoyzpr3+Vx3h6/DF7nU8cdR/e4t9sEtq8bq4qk7uG97fN+/LBx9U1Te6hydU1deBn6L3SX1fkj9J8j3d/DOAd3ddBF+h1y8e4LS+5d7d9/ifZxg/oW98qlvfQXfR6z6Z7gzgjQfX26379Fna3k/vTXY2TwUe362rf72H2gaq6lDbsefgg6p6kN5++Q6AJG/suvC+2tV9UlfDtz23cwbwR33buYteAJ3aLbN/XV/vtnc2P0YvlP9HN/5h4EVJTpmh7fRlF7B32vzp+2xpV9ds29Liy32Pv8Fj9+vhvA6vAF4M3NV1pf3wEdSiBgaGHqOqbqmq8+i98X4WOBg0e4CfmxZCT6yqvzzCVS1P8qS+8dX0usam2wNsnrbe46tqcoa2twGrkkzMss776H0yPqNv2mrgn46g/oNOP/ggyQnACuBL3fmKNwE/CSyvqpOBr9IL2YOm3yp6D/Ciadt6XFX9E71ul/51HU+vu2s26+m9oX4xyZeBP6DXNXXJDG330esGO7js9I/Te12m77OHeeyb+qFue31Ub4ldVXdU1UXA04D/Tu9oRUeBgaFHJTk1ycu7N/Jv0uvWeqSb/T7gF5N8b9f2pCSvHHCVb0ny+O7N9aX03tSmez9wRZIz0/OkJC9J8uTpDatqN/CbwGR63/V4fJLjkrwqyVXVu8zzo8DmJE/uTlD/AvC7A2zDi5Oc1Z1Qvgb466raQ+8cycP0+uaXJvlleud3DuV9XW1nACQ5JclF3bwbgJf2reutzPL3m+Q04IX09ulzu+E5wDvoBcl0fwJ8X5KLkyyld+HA0/vmTwL/Mcm/6kLx7cBHqv1qqLvpnf8Yuu41fnWSk6rqAL1uuEfmep6OjIGxeN2cx34P448anvM4eidnv0Sva+VHgdcCVNUf0XvD+f3uqp5PAy+aZTktvkyvT/9L9LpLrqiqz05vVFU76J2gf2/X/nP0+rhn87qu7W/QOx/zD/S6Z27u5l9J7wT754HtwO8Bvz3AdvwesIne/vpB4ODVXrfQO+/z9/S6cB5i7m6bd9O7cODPknyN3gnwMwGq6jP03sh/j94RwRSP7Tbq9xrgzqr6s6r68sEBeA/w7CTr+htX1X3AK4F30uvmehawg96HBujtn98B/hz4x25brpxjW6Zv1090V0O95zCe1+o1wBe638sr6F1soKMgve5KSYcryQeBvVX15lHXMkzpXWa8F3h1VW0bdT0aHx5hSCLJBUlOTvIE4JfonWv5+IjL0pgxMCQB/DC97rv7gJfRu8run0dbksaNXVKSpCYeYUiSmhgYkqQmBoYkqYmBIUlqYmBIkpoYGJKkJgaGJKmJgSFJamJgSJKaGBiSpCYGhiSpiYEhSWpiYEiSmhgYkqQmS0ddwDA99alPrTVr1oy6DElaUHbu3HlfVZ0yV7tFFRhr1qxhx44doy5DkhaUJHe1tLNLSpLUxMCQJDUxMCRJTQwMSVITA0OS1GSgwEiyIsnWJLu7n8tnabe+a7M7yfq+6ZuT7Eny4LT2T0jykSSfS/LXSdYMUqckaXCDHmFcBdxWVc8EbuvGHyPJCmATcCbwPGBTX7Dc3E2bbgMwVVXfBfw68I4B65QkDWjQwLgIuL57fD1w8QxtLgC2VtX+qpoCtgIXAlTVx6tq3xzLvQF4YZIMWKskaQCDfnHv1INv+FW1L8nTZmhzGrCnb3xvN+1QHn1OVT2c5KvAU4D7pjdMcjlwOcDq1asPewM0vob9GaGqhro86VgzZ2AkuRV4+gyzNjauY6a/+rn+cpufU1XXAdcBTExM+I6wiLS+wScxDKR5MGdgVNW5s81LcneSld3RxUrgnhma7QXO7htfBdw+x2r3AqcDe5MsBU4C9s9VqyTp6Bn0HMZNwMGrntYDN87Q5hbg/CTLu5Pd53fTWpf7E8DHyo+QkjRSgwbGtcB5SXYD53XjJJlI8gGAqtoPXAPc0Q1v7aaR5J1J9gLHJ9mb5OpuuVuApyT5HPALzHD1lSRpfmUxfXCfmJgo71Z77PEchjSYJDuramKudn7TW5LUxMCQJDUxMCRJTQwMSVITA0MjsWLFCpIMZQCGspwVK1aMeK9I421R/U9vLRxTU1Njd2WTtyuTDs0jDElSEwNDktTEwJAkNTEwJElNDAxJUhMDQ5LUxMCQJDUxMCRJTQwMSVITA0OS1MTAkCQ1MTAkSU0MDElSEwNDktTEwJAkNTEwJElNDAxJUhMDQ5LUxMCQJDUxMCRJTQwMSVKTpaMuQJIWoiRDW1ZVDW1ZR5OBIUlHoOVNPsmCCYMWdklJkpoYGJKkJgaGJKmJgSFJajJQYCRZkWRrkt3dz+WztFvftdmdZH3f9M1J9iR5cFr7S5Pcm+TObrhskDolSYMb9Cqpq4DbquraJFd142/qb5BkBbAJmAAK2JnkpqqaAm4G3gvsnmHZH6mq/zBgfRpTtelEuPqkUZfxGLXpxFGXII21QQPjIuDs7vH1wO1MCwzgAmBrVe0HSLIVuBCYrKqPd9MGLEMLTd7ywNhdbpiEunrUVUjja9BzGKdW1T6A7ufTZmhzGrCnb3xvN20ur0jyqSQ3JDl9tkZJLk+yI8mOe++993BqlyQdhjkDI8mtST49w3BR4zpmOnyY66PlzcCaqno2cCu9o5cZVdV1VTVRVROnnHJKY0mSpMM1Z5dUVZ0727wkdydZWVX7kqwE7pmh2V6+1W0FsIpe19Wh1nl/3+j7gXfMVack6egatEvqJuDgVU/rgRtnaHMLcH6S5d1VVOd302bVhc9BLwd2DVinJGlAgwbGtcB5SXYD53XjJJlI8gGA7mT3NcAd3fDWvhPg70yyFzg+yd4kV3fLfV2SzyT5JPA64NIB65QkDSjjdqXKICYmJmrHjh2jLkMNxvGmbONYkxa2hfI7lWRnVU3M1c5vekuSmnh7c+kYMezvOy2ET84aLgNDOkYci/+/QcNll5QkqYmBIUlqYmBIkpoYGJKkJgaGJKmJV0lJUp8VK1YwNTU1tOUN63Lm5cuXs3///qEs60gZGJLUZ2pqaiwvLR6H/xtkl5QkqYmBIUlqYmBIkpoYGJKkJgaGJKmJgSFJamJgSJKaGBiSpCYGhiSpiYEhSWrirUGkBc57H2m+GBjSAue9jzRf7JKSJDUxMCRJTQwMSVITA0OS1MTAkCQ1MTAkSU0MDElSE7+HoZEZt+v0ly9fPuoSpLFmYGgkhvlFsyRj+cU1abGxS0qS1GSgI4wkK4CPAGuALwA/WVXfdlObJOuBN3ejb6uq65McD/wB8J3AI8DNVXVV1/4JwIeAHwTuB36qqr4wSK3SYlWbToSrTxp1Gd+mNp046hKOiPtzdhnkUD7JO4H9VXVtkquA5VX1pmltVgA7gAmggJ30guCbwJlVtS3J44HbgLdX1Z8meS3w7Kq6IsmrgB+rqp+aq56JiYnasWPHEW+PFqZjvUtqXLd/XOuay7jWfTTrSrKzqibmajdol9RFwPXd4+uBi2docwGwtar2d0cfW4ELq+obVbUNoKr+BfgEsGqG5d4AvDDjdoZUko4xgwbGqVW1D6D7+bQZ2pwG7Okb39tNe1SSk4GX0TvKeMxzquph4KvAUwasVZI0gDnPYSS5FXj6DLM2Nq5jpiODR4+rkiwFJoH3VNXnW54zrb7LgcsBVq9e3ViSJOlwzRkYVXXubPOS3J1kZVXtS7ISuGeGZnuBs/vGVwG3941fB+yuqndNe87pwN4uUE4CZvxPLFV1XbcMJiYmxq/jUdKCM4494OPwPaFBu6RuAtZ3j9cDN87Q5hbg/CTLkywHzu+mkeRt9MLgDYdY7k8AH6txPAsladGpqqENw1zeOPz3wkED41rgvCS7gfO6cZJMJPkAQFXtB64B7uiGt1bV/iSr6HVrPQv4RJI7k1zWLXcL8JQknwN+AbhqwDolSQMa6LLaceNltcemcb0Mcr6M6/aPa13zaaHsg/m6rFaSdIwwMCRJTQwMSVIT71YrSUeg9dLblnYL4TwHGBiSdEQWypv8MNklJUlqYmBIkpoYGJKkJgaGJKmJgSFJamJgSJKaeFmttAh4O27NBwNDWuCG+X2AhXKzPI2GXVKSpCYGhiSpiV1SGluH0y+/mO7XI40rA0Njyzd4abzYJSVJamJgSJKaGBiSpCYGhiSpiYEhSWpiYEiSmhgYkqQmBoYkqYmBIUlqYmBIkpoYGJKkJt5LasiG+Y9svJeSpHFiYAxZy5u8/6RG0kJkl5QkqYmBIUlqYmBIkpoMFBhJViTZmmR393P5LO3Wd212J1nfTTs+yZ8k+WySzyS5tq/9pUnuTXJnN1w2SJ2SpMENeoRxFXBbVT0TuK0bf4wkK4BNwJnA84BNfcHyK1X1PcD3A89P8qK+p36kqp7bDR8YsE5J0oAGDYyLgOu7x9cDF8/Q5gJga1Xtr6opYCtwYVV9o6q2AVTVvwCfAFYNWI8k6SgZNDBOrap9AN3Pp83Q5jRgT9/43m7ao5KcDLyM3lHKQa9I8qkkNyQ5fcA6JUkDmvN7GEluBZ4+w6yNjeuY6Ztsj34JIclSYBJ4T1V9vpt8MzBZVd9McgW9o5cXzFLf5cDlAKtXr24sSZJ0uOYMjKo6d7Z5Se5OsrKq9iVZCdwzQ7O9wNl946uA2/vGrwN2V9W7+tZ5f9/89wPvOER913XLYGJiwm/DSdJRMmiX1E3A+u7xeuDGGdrcApyfZHl3svv8bhpJ3gacBLyh/wld+Bz0cmDXgHVKx7wkcw6t7YZ5CxwtHIPeGuRa4KNJNgBfBF4JkGQCuKKqLquq/UmuAe7onvPWbtoqet1anwU+0f0Cvre7Iup1SV4OPAzsBy4dsE7pmOftaDSoLKZfoomJidqxY8eoy5iT95KSNE6S7Kyqibna+U1vSVITA0OS1MTAkCQ1MTAkSU0MDElSEwNDktTEwJAkNTEwJElNDAxJUhMDQ5LUxMCQJDUxMCRJTQwMSVKTQW9vfsxYsWIFU1NTQ1vesP6fwPLly9m/f/9QliVJh2JgNJqamhrLW5L7j2wkzRe7pCRJTQwMSVITA0OS1MTAkCQ1MTAkMTk5ybp161iyZAnr1q1jcnJy1CVpDHmVlHSMm5ycZOPGjWzZsoWzzjqL7du3s2HDBgAuueSSEVenceIRhnSM27x5M1u2bOGcc85h2bJlnHPOOWzZsoXNmzePujSNmYzjdwuO1MTERO3YseOoLDvJ2H4PYxzr0sKxZMkSHnroIZYtW/botAMHDnDcccfxyCOPjLAyzZckO6tqYq52HmFIx7i1a9eyffv2x0zbvn07a9euHVFFGlcGhnSM27hxIxs2bGDbtm0cOHCAbdu2sWHDBjZu3Djq0jRmPOktHeMOnti+8sor2bVrF2vXrmXz5s2e8Na38RxGo3E9VzCudUlaODyHIUkaKgNDktTEwJAkNTEwJElNDAxJUhMDQ5LUZODASLIiydYku7ufy2dpt75rszvJ+r7p/zPJJ5N8Jsn7kiw5nOVKkubHMI4wrgJuq6pnArd144+RZAWwCTgTeB6wqS8AfrKqngOsA04BXtm6XEnS/BlGYFwEXN89vh64eIY2FwBbq2p/VU0BW4ELAarqga7NUuDxwMFvobUsV5I0T4YRGKdW1T6A7ufTZmhzGrCnb3xvNw2AJLcA9wBfA244jOVKkuZJ072kktwKPH2GWa13J8sM0x69n0VVXZDkOODDwAvoHYG0LTi5HLgcYPXq1a1PkyQdpqbAqKpzZ5uX5O4kK6tqX5KV9I4UptsLnN03vgq4fdo6HkpyE72uqK1Ay3KpquuA66B3L6mW7TkStelEuPqko7X4I1abThx1CZKOEcO4W+1NwHrg2u7njTO0uQV4e9+J7vOBX0xyAvDkLhSWAi8G/vdhLHfe5C0PjOVN/pJQV4+6CknHgmGcw7gWOC/JbuC8bpwkE0k+AFBV+4FrgDu64a3dtCcBNyX5FPBJekcR7zvUciVJo+HtzRuN623Ex7UuSQuHtzeXJA2VgSFJamJgSJKaGBiSpCbDuKz2mJHM9P3D0Vq+3HsySpofBkajYV6J5JVNkhYiu6QkSU0MDElSEwNDktTEwJAkNTEwJElNDAxJUhMDQ5LUxMCQJDUxMCRpyCYnJ1m3bh1Llixh3bp1TE5OjrqkofCb3pI0RJOTk2zcuJEtW7Zw1llnsX37djZs2ADAJZdcMuLqBuMRhiQN0ebNm9myZQvnnHMOy5Yt45xzzmHLli1s3rx51KUNzP+4NwLeS0pavJYsWcJDDz3EsmXLHp124MABjjvuOB555JERVjY7/+OeJI3A2rVr2b59+2Ombd++nbVr146oouExMCRpiDZu3MiGDRvYtm0bBw4cYNu2bWzYsIGNGzeOurSBedJbkobo4IntK6+8kl27drF27Vo2b9684E94g+cwRsJzGJLGiecwJElDZWBIkpoYGJKkJp70HrIkQ2vneQ5J48TAGDLf5CUtVnZJSZKaGBiSpCYGhiSpiYEhSWpiYEiSmhgYkqQmBoYkqYmBIUlqsqjuVpvkXuCuUdfR4KnAfaMuYhFxfw6P+3K4Fsr+PKOqTpmr0aIKjIUiyY6WWwmrjftzeNyXw7XY9qddUpKkJgaGJKmJgTEa1426gEXG/Tk87svhWlT703MYkqQmHmFIkpoYGPMoyW8nuSfJp0ddy0KX5PQk25LsSvKZJK8fdU0LWZLjkvxNkk92+/Mto65poUuyJMn/SfLHo65lWAyM+fVB4MJRF7FIPAy8sarWAj8E/HySZ424poXsm8ALquo5wHOBC5P80IhrWuheD+wadRHDZGDMo6r6c2D/qOtYDKpqX1V9onv8NXp/mKeNtqqFq3oe7EaXdYMnOI9QklXAS4APjLqWYTIwtOAlWQN8P/DXo61kYeu6UO4E7gG2VpX788i9C/jPwP8bdSHDZGBoQUtyAvDfgDdU1QOjrmchq6pHquq5wCrgeUnWjbqmhSjJS4F7qmrnqGsZNgNDC1aSZfTC4sNV9YejrmexqKqvALfj+bYj9Xzg5Um+APw+8IIkvzvakobDwNCClCTAFmBXVf3aqOtZ6JKckuTk7vETgXOBz462qoWpqn6xqlZV1RrgVcDHquqnR1zWUBgY8yjJJPBXwHcn2Ztkw6hrWsCeD7yG3qe3O7vhxaMuagFbCWxL8ingDnrnMBbN5aAaDr/pLUlq4hGGJKmJgSFJamJgSJKaGBiSpCYGhiSpiYEhSWpiYEiSmhgYkqQm/x/hwzSEYfTiZgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Ensemble and Boosting algorithm to improve performance \n",
    "from sklearn.ensemble import AdaBoostRegressor \n",
    "from sklearn.ensemble import GradientBoostingRegressor \n",
    "from sklearn.ensemble import RandomForestRegressor \n",
    "from sklearn.ensemble import ExtraTreesRegressor \n",
    "\n",
    "ensembles = []\n",
    "ensembles.append(('scaledAB', Pipeline([('scale', StandardScaler()), ('AB', AdaBoostRegressor())])))\n",
    "ensembles.append(('scaledGBR', Pipeline([('scale', StandardScaler()), ('GBR', GradientBoostingRegressor())])))\n",
    "ensembles.append(('scaledRF', Pipeline([('scale', StandardScaler()), ('RF', RandomForestRegressor())])))\n",
    "ensembles.append(('scaledETR', Pipeline([('scale', StandardScaler()), ('ETR', ExtraTreesRegressor())])))\n",
    "\n",
    "results = []\n",
    "names = []\n",
    "for name, model in ensembles:\n",
    "    fold = KFold(n_splits=10, random_state=5)\n",
    "    result = cross_val_score(model, x_train, y_train, cv=fold, scoring=error)\n",
    "    results.append(result)\n",
    "    names.append(name)\n",
    "    msg = '%s : %f (%f)'% (name, result.mean(), result.std())\n",
    "    print(msg)\n",
    "\n",
    "fig = plt.figure()\n",
    "fig.suptitle('Ensemble Compared Algorithms')\n",
    "plt.boxplot(results)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# ExtraTreesRegressor Tuning \n",
    "scaler = StandardScaler().fit(x_train)\n",
    "rescaledx = scaler.transform(x_train)\n",
    "\n",
    "n_estimators = [5, 10, 20, 30, 40, 50, 100, 200]\n",
    "param_grid = dict(n_estimators=n_estimators)\n",
    "model = ExtraTreesRegressor()\n",
    "fold = KFold(n_splits=10, random_state=5)\n",
    "grid_etr = GridSearchCV(estimator=model, param_grid=param_grid, scoring=error-, cv=fold)\n",
    "grid_etr = grid_etr.fit(x_train, y_train)\n",
    "\n",
    "# print('Best: %f using %s' % (grid_result.best_score_, grid_result.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.017118454433971274\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([113766.19226501, 152389.79076575, 174738.45604329, ...,\n",
       "       146615.91406782, 112115.41260962, 226064.29725796])"
      ]
     },
     "execution_count": 259,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(grid_etr.best_score_)\n",
    "\n",
    "etr_pred = np.expm1(grid_etr.best_estimator_.predict(x_test))\n",
    "etr_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.13083751157054033"
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sqrt(0.017118454433971274)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.014804 using: {'learning_rate': 0.2, 'n_estimators': 200}\n"
     ]
    }
   ],
   "source": [
    "# GradientBoostingRegressor Tuning \n",
    "scaler = StandardScaler().fit(x_train)\n",
    "rescaledx = scaler.transform(x_train)\n",
    "\n",
    "learning_rate = [0.1, 0.2, 0.3, 0.4, 0.5]\n",
    "n_estimators = [5, 10, 20, 30, 40, 50, 100, 200]\n",
    "param_grid = dict(n_estimators=n_estimators, learning_rate=learning_rate)\n",
    "\n",
    "model = GradientBoostingRegressor()\n",
    "fold = KFold(n_splits=10, random_state=5)\n",
    "grid = GridSearchCV(estimator=model, param_grid=param_grid, scoring=error, cv=fold)\n",
    "grid_result = grid.fit(rescaledx, y_train)\n",
    "\n",
    "print(\"Best: %f using: %s\" % (grid_result.best_score_, grid_result.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_gbr = grid.best_estimator_.fit(x_train, y_train)\n",
    "gbr_pred = np.expm1(grid_gbr.predict(x_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Finalize Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error \n",
    "\n",
    "scaler = StandardScaler().fit(x_train)\n",
    "scaler_x = scaler.transform(x_train)\n",
    "model = GradientBoostingRegressor(random_state=5, n_estimators=200, learning_rate=0.2)\n",
    "model.fit(scaler_x, y_train)\n",
    "\n",
    "scaledx_test = scaler.transform(x_test)\n",
    "y_pred = model.predict(scaledx_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.DataFrame()\n",
    "sub['Id'] = test_id \n",
    "sub['SalePrice'] = np.expm1(y_pred)*1.01\n",
    "sub.to_csv('submission_2020_02_29.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lightgbm\n",
    "import lightgbm as lgb \n",
    "\n",
    "paramaters = {\n",
    "    'max_depth': [4],\n",
    "    'num_leaves': [3],\n",
    "    'feature_fraction': [0.2],\n",
    "    'cat_smmoth': [1],\n",
    "    'bagging_fraction': [0.9],\n",
    "    'bagging_fraq': [3],\n",
    "    'cat_smooth': [0]\n",
    "}\n",
    "\n",
    "gbm = lgb.LGBMRegressor(\n",
    "                objective = 'regression',\n",
    "                learning_rate = 0.01,\n",
    "                n_estimators = 5000,\n",
    "                verbose = -1\n",
    ")\n",
    "\n",
    "gscv = GridSearchCV(gbm,\n",
    "                   param_grid = paramaters,\n",
    "                   cv = 5,\n",
    "                   n_jobs = -1)\n",
    "\n",
    "gscv.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gscv.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgb_pred = np.expm1(gscv.best_estimator_.predict(x_test))\n",
    "\n",
    "sub = pd.DataFrame()\n",
    "sub['Id'] = test_id \n",
    "sub['SalePrice'] = 0.7*lgb_pred + 0.2*gbr_pred + 0.1*etr_pred\n",
    "sub.to_csv('submission_2020_02_29.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
